package com.flink.datastreamapi.sink;

import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.serialization.SimpleStringEncoder;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.connector.source.util.ratelimit.RateLimiterStrategy;
import org.apache.flink.configuration.MemorySize;
import org.apache.flink.connector.datagen.source.DataGeneratorSource;
import org.apache.flink.connector.datagen.source.GeneratorFunction;
import org.apache.flink.connector.file.sink.FileSink;
import org.apache.flink.core.fs.Path;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.filesystem.OutputFileConfig;
import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.DateTimeBucketAssigner;
import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy;

import java.time.Duration;
import java.time.ZoneId;

public class FileSinkDemo {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();


        env.setParallelism(1);

        // 如果设置了滚动策略，必须开启checkpoint，否则一直都是 .inprogress。
      env.enableCheckpointing(2000, CheckpointingMode.EXACTLY_ONCE);

       //数据源
        DataGeneratorSource<String> dataGeneratorSource = new DataGeneratorSource<>(
                new GeneratorFunction<Long, String>() {
                    @Override
                    public String map(Long value) throws Exception {
                        return "Number:" + value;
                    }
                },
                Long.MAX_VALUE,
                //速率限制为每秒生成处理100 个
                RateLimiterStrategy.perSecond(100),
                Types.STRING
        );


        //获取数据源
        DataStreamSource<String> dataGen = env.fromSource(dataGeneratorSource, WatermarkStrategy.noWatermarks(), "data-generator");

        // 输出到文件系统
        FileSink<String> fileSink = FileSink
                //输出行式存储的文件，指定路径、指定编码
                .<String>forRowFormat(new Path("F:/tmp12"),
                                new SimpleStringEncoder<>("UTF-8"))
                // 输出文件的一些配置(选做)： 文件名的前缀、后缀
                .withOutputFileConfig(OutputFileConfig.builder()
                                      .withPartPrefix("flink-")
                                      .withPartSuffix(".log")
                                      .build())
                //按照目录分桶(选做)：如下，就是每个小时一个目录, ZoneId.systemDefault()表示默认时区
               .withBucketAssigner(new DateTimeBucketAssigner<>("yyyy-MM-dd HH", ZoneId.systemDefault()))
                // 文件滚动策略（重要）: 10 或 1KB
                .withRollingPolicy(DefaultRollingPolicy.builder()
                        //间隔时间： 如ofSeconds(10)表示间隔10秒钟
                        .withRolloverInterval(Duration.ofSeconds(10))
                        //定义大小1KB
                        .withMaxPartSize(new MemorySize(1024))
                        .build()
                )
                .build();

        //输出到文件
        dataGen.sinkTo(fileSink);

        env.execute();
    }
}
